{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bf1c2fba-9e47-432f-8e2f-13433d129e45",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-29T22:57:32.696498Z",
     "iopub.status.busy": "2022-06-29T22:57:32.696251Z",
     "iopub.status.idle": "2022-06-29T22:57:36.033738Z",
     "shell.execute_reply": "2022-06-29T22:57:36.033129Z",
     "shell.execute_reply.started": "2022-06-29T22:57:32.696472Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ClientContext(dashboard_url='127.0.0.1:8265', python_version='3.7.10', ray_version='1.13.0', ray_commit='e4ce38d001dbbe09cd21c497fedd03d692b2be3e', protocol_version='2022-03-16', _num_clients=1, _context_to_restore=<ray.util.client._ClientContext object at 0x7facf2418350>)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ray\n",
    "\n",
    "ray.init(\"ray://localhost:10001\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d3873ec3-ff53-4d5e-ab4a-4957f4c6907d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-29T23:00:28.607959Z",
     "iopub.status.busy": "2022-06-29T23:00:28.607718Z",
     "iopub.status.idle": "2022-06-29T23:00:28.616841Z",
     "shell.execute_reply": "2022-06-29T23:00:28.616224Z",
     "shell.execute_reply.started": "2022-06-29T23:00:28.607933Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'node:172.31.0.194': 1.0,\n",
       " 'memory': 17431498344.0,\n",
       " 'object_store_memory': 7987059916.0,\n",
       " 'node:172.31.13.216': 1.0,\n",
       " 'CPU': 8.0}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Before cluster resize\n",
    "ray.cluster_resources()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a8c25e95-93c3-4331-8003-758a7fddff73",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-29T23:19:53.421175Z",
     "iopub.status.busy": "2022-06-29T23:19:53.420906Z",
     "iopub.status.idle": "2022-06-29T23:19:55.088376Z",
     "shell.execute_reply": "2022-06-29T23:19:55.087380Z",
     "shell.execute_reply.started": "2022-06-29T23:19:53.421146Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Resizing ig-1VN1F0JIC2H1P on j-FR2PRWRFQ5BZ\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "An error occurred (AccessDeniedException) when calling the ModifyInstanceGroups operation: User: arn:aws:sts::568026268536:assumed-role/EMR_EC2_DefaultRole/i-0a49a4c9676ee2ddb is not authorized to perform: elasticmapreduce:ModifyInstanceGroups on resource: * because no identity-based policy allows the elasticmapreduce:ModifyInstanceGroups action\n"
     ]
    },
    {
     "ename": "CalledProcessError",
     "evalue": "Command 'b'\\nREGION=$(curl --silent http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)\\nINSTANCE=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`\\nCLUSTER_ID=$(aws --region ${REGION} ec2 describe-instances --instance-ids $INSTANCE --query \\'Reservations[0].Instances[0].Tags[?Key==`aws:elasticmapreduce:job-flow-id`].Value\\' | jq -r \\'.[0]\\')\\nCORE_INSTANCE_GROUP=$(aws --region ${REGION} emr list-instances --cluster-id ${CLUSTER_ID} --instance-group-types \"CORE\" | jq -r \\'.Instances[0].InstanceGroupId\\')\\n\\necho \"Resizing ${CORE_INSTANCE_GROUP} on ${CLUSTER_ID}\"\\n\\naws --region ${REGION} emr modify-instance-groups --instance-groups InstanceGroupId=${CORE_INSTANCE_GROUP},InstanceCount=2\\n'' returned non-zero exit status 255.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mCalledProcessError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-f8de50ed2283>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'sh'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\nREGION=$(curl --silent http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)\\nINSTANCE=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`\\nCLUSTER_ID=$(aws --region ${REGION} ec2 describe-instances --instance-ids $INSTANCE --query \\'Reservations[0].Instances[0].Tags[?Key==`aws:elasticmapreduce:job-flow-id`].Value\\' | jq -r \\'.[0]\\')\\nCORE_INSTANCE_GROUP=$(aws --region ${REGION} emr list-instances --cluster-id ${CLUSTER_ID} --instance-group-types \"CORE\" | jq -r \\'.Instances[0].InstanceGroupId\\')\\n\\necho \"Resizing ${CORE_INSTANCE_GROUP} on ${CLUSTER_ID}\"\\n\\naws --region ${REGION} emr modify-instance-groups --instance-groups InstanceGroupId=${CORE_INSTANCE_GROUP},InstanceCount=2\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/emr/notebook-env/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2417\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2418\u001b[0m                 \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2419\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2420\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/emr/notebook-env/lib/python3.7/site-packages/IPython/core/magics/script.py\u001b[0m in \u001b[0;36mnamed_script_magic\u001b[0;34m(line, cell)\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m                 \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscript\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshebang\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0;31m# write a basic docstring:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/emr/notebook-env/lib/python3.7/site-packages/decorator.py\u001b[0m in \u001b[0;36mfun\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m    230\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mkwsyntax\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcaller\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextras\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m     \u001b[0mfun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m     \u001b[0mfun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/emr/notebook-env/lib/python3.7/site-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    185\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/emr/notebook-env/lib/python3.7/site-packages/IPython/core/magics/script.py\u001b[0m in \u001b[0;36mshebang\u001b[0;34m(self, line, cell)\u001b[0m\n\u001b[1;32m    243\u001b[0m             \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_error\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m!=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mCalledProcessError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_script\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mto_close\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mCalledProcessError\u001b[0m: Command 'b'\\nREGION=$(curl --silent http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)\\nINSTANCE=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`\\nCLUSTER_ID=$(aws --region ${REGION} ec2 describe-instances --instance-ids $INSTANCE --query \\'Reservations[0].Instances[0].Tags[?Key==`aws:elasticmapreduce:job-flow-id`].Value\\' | jq -r \\'.[0]\\')\\nCORE_INSTANCE_GROUP=$(aws --region ${REGION} emr list-instances --cluster-id ${CLUSTER_ID} --instance-group-types \"CORE\" | jq -r \\'.Instances[0].InstanceGroupId\\')\\n\\necho \"Resizing ${CORE_INSTANCE_GROUP} on ${CLUSTER_ID}\"\\n\\naws --region ${REGION} emr modify-instance-groups --instance-groups InstanceGroupId=${CORE_INSTANCE_GROUP},InstanceCount=2\\n'' returned non-zero exit status 255."
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "\n",
    "# I was hoping to do a manual resize here, but ala\n",
    "REGION=$(curl --silent http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)\n",
    "INSTANCE=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`\n",
    "CLUSTER_ID=$(aws --region ${REGION} ec2 describe-instances --instance-ids $INSTANCE --query 'Reservations[0].Instances[0].Tags[?Key==`aws:elasticmapreduce:job-flow-id`].Value' | jq -r '.[0]')\n",
    "CORE_INSTANCE_GROUP=$(aws --region ${REGION} emr list-instances --cluster-id ${CLUSTER_ID} --instance-group-types \"CORE\" | jq -r '.Instances[0].InstanceGroupId')\n",
    "\n",
    "echo \"Resizing ${CORE_INSTANCE_GROUP} on ${CLUSTER_ID}\"\n",
    "\n",
    "aws --region ${REGION} emr modify-instance-groups --instance-groups InstanceGroupId=${CORE_INSTANCE_GROUP},InstanceCount=2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6574dffe-d398-423f-bb97-0ff7770f16a9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-28T20:42:36.170291Z",
     "iopub.status.busy": "2022-06-28T20:42:36.170046Z",
     "iopub.status.idle": "2022-06-28T20:42:36.179473Z",
     "shell.execute_reply": "2022-06-28T20:42:36.178747Z",
     "shell.execute_reply.started": "2022-06-28T20:42:36.170265Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'memory': 28497285941.0,\n",
       " 'node:172.31.9.40': 1.0,\n",
       " 'CPU': 12.0,\n",
       " 'object_store_memory': 12738014821.0,\n",
       " 'node:172.31.10.50': 1.0,\n",
       " 'node:172.31.9.84': 1.0}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# After cluster resize\n",
    "ray.cluster_resources()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "63d322a3-e578-4eda-9601-197f1bee2d7e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-29T22:57:15.973290Z",
     "iopub.status.busy": "2022-06-29T22:57:15.973055Z",
     "iopub.status.idle": "2022-06-29T22:57:16.970179Z",
     "shell.execute_reply": "2022-06-29T22:57:16.969402Z",
     "shell.execute_reply.started": "2022-06-29T22:57:15.973265Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b161d162-ff8c-4bc2-84b6-cd2cc00c597e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-29T22:59:12.154839Z",
     "iopub.status.busy": "2022-06-29T22:59:12.154506Z",
     "iopub.status.idle": "2022-06-29T22:59:20.717797Z",
     "shell.execute_reply": "2022-06-29T22:59:20.717065Z",
     "shell.execute_reply.started": "2022-06-29T22:59:12.154803Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 22:59:16,120\tWARNING pool.py:591 -- The 'context' argument is not supported using ray. Please refer to the documentation for how to control ray initialization.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 5 candidates, totalling 10 fits\n",
      "\u001b[2m\u001b[36m(PoolActor pid=23237, ip=172.31.0.194)\u001b[0m [CV 2/2; 1/5] START C=2.592943797404667e-06, class_weight=None, gamma=3.562247890262444e-08, tol=0.0002592943797404667\n",
      "\u001b[2m\u001b[36m(PoolActor pid=23238, ip=172.31.0.194)\u001b[0m [CV 1/2; 4/5] START C=1268.9610031679233, class_weight=None, gamma=0.0009236708571873865, tol=0.005736152510448681\n",
      "\u001b[2m\u001b[36m(PoolActor pid=23237, ip=172.31.0.194)\u001b[0m [CV 2/2; 1/5] END C=2.592943797404667e-06, class_weight=None, gamma=3.562247890262444e-08, tol=0.0002592943797404667;, score=0.352 total time=   0.2s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=23237, ip=172.31.0.194)\u001b[0m [CV 2/2; 5/5] START C=8531.678524172814, class_weight=balanced, gamma=5.736152510448681e-06, tol=0.02395026619987486\n",
      "\u001b[2m\u001b[36m(PoolActor pid=23238, ip=172.31.0.194)\u001b[0m [CV 1/2; 4/5] END C=1268.9610031679233, class_weight=None, gamma=0.0009236708571873865, tol=0.005736152510448681;, score=0.959 total time=   0.1s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10110)\u001b[0m [CV 2/2; 2/5] START C=0.00011721022975334806, class_weight=balanced, gamma=0.041753189365604, tol=0.0002592943797404667\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10207)\u001b[0m [CV 1/2; 3/5] START C=0.0020433597178569417, class_weight=balanced, gamma=85.31678524172814, tol=0.04893900918477494\n",
      "\u001b[2m\u001b[36m(PoolActor pid=23237, ip=172.31.0.194)\u001b[0m [CV 2/2; 5/5] END C=8531.678524172814, class_weight=balanced, gamma=5.736152510448681e-06, tol=0.02395026619987486;, score=0.948 total time=   0.1s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10105)\u001b[0m [CV 1/2; 2/5] START C=0.00011721022975334806, class_weight=balanced, gamma=0.041753189365604, tol=0.0002592943797404667\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10107)\u001b[0m [CV 1/2; 1/5] START C=2.592943797404667e-06, class_weight=None, gamma=3.562247890262444e-08, tol=0.0002592943797404667\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10106)\u001b[0m [CV 2/2; 4/5] START C=1268.9610031679233, class_weight=None, gamma=0.0009236708571873865, tol=0.005736152510448681\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10209)\u001b[0m [CV 2/2; 3/5] START C=0.0020433597178569417, class_weight=balanced, gamma=85.31678524172814, tol=0.04893900918477494\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10106)\u001b[0m [CV 2/2; 4/5] END C=1268.9610031679233, class_weight=None, gamma=0.0009236708571873865, tol=0.005736152510448681;, score=0.965 total time=   0.2s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10207)\u001b[0m [CV 1/2; 3/5] END C=0.0020433597178569417, class_weight=balanced, gamma=85.31678524172814, tol=0.04893900918477494;, score=0.099 total time=   0.5s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10110)\u001b[0m [CV 2/2; 2/5] END C=0.00011721022975334806, class_weight=balanced, gamma=0.041753189365604, tol=0.0002592943797404667;, score=0.097 total time=   0.6s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10105)\u001b[0m [CV 1/2; 2/5] END C=0.00011721022975334806, class_weight=balanced, gamma=0.041753189365604, tol=0.0002592943797404667;, score=0.101 total time=   0.6s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10107)\u001b[0m [CV 1/2; 1/5] END C=2.592943797404667e-06, class_weight=None, gamma=3.562247890262444e-08, tol=0.0002592943797404667;, score=0.101 total time=   0.5s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10107)\u001b[0m [CV 1/2; 5/5] START C=8531.678524172814, class_weight=balanced, gamma=5.736152510448681e-06, tol=0.02395026619987486\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10209)\u001b[0m [CV 2/2; 3/5] END C=0.0020433597178569417, class_weight=balanced, gamma=85.31678524172814, tol=0.04893900918477494;, score=0.098 total time=   0.4s\n",
      "\u001b[2m\u001b[36m(PoolActor pid=10107)\u001b[0m [CV 1/2; 5/5] END C=8531.678524172814, class_weight=balanced, gamma=5.736152510448681e-06, tol=0.02395026619987486;, score=0.932 total time=   0.1s\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.svm import SVC\n",
    "import ray\n",
    "ray.shutdown()\n",
    "\n",
    "import joblib\n",
    "from ray.util.joblib import register_ray\n",
    "\n",
    "register_ray()\n",
    "#ray.init(address=\"localhost:6379\")\n",
    "ray.init(\"ray://localhost:10001\")\n",
    "\n",
    "#import mlflow\n",
    "#mlflow.set_tracking_uri('http://127.0.0.1:5001')\n",
    "# mlflow.autolog() #log_models=False, exclusive=True)\n",
    "import sklearn\n",
    "#mlflow.sklearn.autolog()#log_models=True)\n",
    "\n",
    "digits = load_digits()\n",
    "\n",
    "param_space = {\n",
    "    'C': np.logspace(-6, 6, 30),\n",
    "    'gamma': np.logspace(-8, 8, 30),\n",
    "    'tol': np.logspace(-4, -1, 30),\n",
    "    'class_weight': [None, 'balanced'],\n",
    "}\n",
    "model = SVC(kernel='rbf')\n",
    "search = RandomizedSearchCV(model, param_space, cv=2, n_iter=5, verbose=10)\n",
    "\n",
    "with joblib.parallel_backend('ray'):\n",
    "    search.fit(digits.data, digits.target)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
